const { fetchPage } = require('./modules/crawler/fetch');
const { parseNewsList, parseArticle } = require('./modules/crawler/parse');
const { canCrawl } = require('./modules/crawler/robots');
const { saveNews, checkUrlExists } = require('./modules/storage/db');
const { downloadImage } = require('./modules/storage/images');
const RateLimiter = require('./modules/utils/rate-limit');

const BASE_URL = 'https://news.qq.com';
const CATEGORIES = [
  { url: '/society', name: '社会' },
  { url: '/politics', name: '时政' },
  { url: '/economy', name: '财经' },
  { url: '/technology', name: '科技' }
];

async function crawlNews() {
  const rateLimiter = new RateLimiter({ interval: 2000 }); // 2秒间隔
  console.log('开始爬取新闻...');
  
  for (const category of CATEGORIES) {
    const categoryUrl = `${BASE_URL}${category.url}`;
    
    try {
      // 检查是否允许爬取
      const allowed = await canCrawl(categoryUrl);
      if (!allowed) {
        console.log(`禁止爬取: ${categoryUrl}`);
        continue;
      }
      
      await rateLimiter.wait();
      const html = await fetchPage(categoryUrl);
      const newsItems = parseNewsList(html);
      
      console.log(`从 ${category.name} 分类获取到 ${newsItems.length} 条新闻`);
      
      for (const news of newsItems) {
        // 检查是否已爬取过
        const exists = await checkUrlExists(news.url);
        if (exists) continue;
        
        // 下载图片
        if (news.imageUrl) {
          try {
            await rateLimiter.wait();
            const imagePath = await downloadImage(news.imageUrl);
            news.imagePath = imagePath;
          } catch (error) {
            console.error(`下载图片失败: ${news.imageUrl}`, error.message);
          }
        }
        
        // 爬取文章详情
        try {
          await rateLimiter.wait();
          const articleUrl = news.url.startsWith('http') ? news.url : `${BASE_URL}${news.url}`;
          const articleHtml = await fetchPage(articleUrl);
          const article = parseArticle(articleHtml);
          news.content = article.content;
          news.author = article.author;
        } catch (error) {
          console.error(`获取文章详情失败: ${news.url}`, error.message);
        }
        
        // 保存到数据库
        try {
          const newsId = await saveNews(news);
          console.log(`成功保存新闻: ${news.title} (ID: ${newsId})`);
        } catch (error) {
          console.error(`保存新闻失败: ${news.title}`, error);
        }
      }
    } catch (error) {
      console.error(`爬取分类 ${category.name} 失败`, error);
    }
  }
  
  console.log('新闻爬取完成!');
}

// 启动爬虫
crawlNews().catch(error => {
  console.error('爬虫运行出错', error);
  process.exit(1);
});  